{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pregnants</th>\n",
       "      <th>Plasma_glucose_concentration</th>\n",
       "      <th>blood_pressure</th>\n",
       "      <th>Triceps_skin_fold_thickness</th>\n",
       "      <th>serum_insulin</th>\n",
       "      <th>BMI</th>\n",
       "      <th>Diabetes_pedigree_function</th>\n",
       "      <th>Age</th>\n",
       "      <th>Target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.639947</td>\n",
       "      <td>0.848324</td>\n",
       "      <td>0.149641</td>\n",
       "      <td>0.907270</td>\n",
       "      <td>-0.692891</td>\n",
       "      <td>0.204013</td>\n",
       "      <td>0.468492</td>\n",
       "      <td>1.425995</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.844885</td>\n",
       "      <td>-1.123396</td>\n",
       "      <td>-0.160546</td>\n",
       "      <td>0.530902</td>\n",
       "      <td>-0.692891</td>\n",
       "      <td>-0.684422</td>\n",
       "      <td>-0.365061</td>\n",
       "      <td>-0.190672</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.233880</td>\n",
       "      <td>1.943724</td>\n",
       "      <td>-0.263941</td>\n",
       "      <td>-1.288212</td>\n",
       "      <td>-0.692891</td>\n",
       "      <td>-1.103255</td>\n",
       "      <td>0.604397</td>\n",
       "      <td>-0.105584</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.844885</td>\n",
       "      <td>-0.998208</td>\n",
       "      <td>-0.160546</td>\n",
       "      <td>0.154533</td>\n",
       "      <td>0.123302</td>\n",
       "      <td>-0.494043</td>\n",
       "      <td>-0.920763</td>\n",
       "      <td>-1.041549</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1.141852</td>\n",
       "      <td>0.504055</td>\n",
       "      <td>-1.504687</td>\n",
       "      <td>0.907270</td>\n",
       "      <td>0.765836</td>\n",
       "      <td>1.409746</td>\n",
       "      <td>5.484909</td>\n",
       "      <td>-0.020496</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   pregnants  Plasma_glucose_concentration  blood_pressure  \\\n",
       "0   0.639947                      0.848324        0.149641   \n",
       "1  -0.844885                     -1.123396       -0.160546   \n",
       "2   1.233880                      1.943724       -0.263941   \n",
       "3  -0.844885                     -0.998208       -0.160546   \n",
       "4  -1.141852                      0.504055       -1.504687   \n",
       "\n",
       "   Triceps_skin_fold_thickness  serum_insulin       BMI  \\\n",
       "0                     0.907270      -0.692891  0.204013   \n",
       "1                     0.530902      -0.692891 -0.684422   \n",
       "2                    -1.288212      -0.692891 -1.103255   \n",
       "3                     0.154533       0.123302 -0.494043   \n",
       "4                     0.907270       0.765836  1.409746   \n",
       "\n",
       "   Diabetes_pedigree_function       Age  Target  \n",
       "0                    0.468492  1.425995       1  \n",
       "1                   -0.365061 -0.190672       0  \n",
       "2                    0.604397 -0.105584       1  \n",
       "3                   -0.920763 -1.041549       0  \n",
       "4                    5.484909 -0.020496       1  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv(\"./data/FE_diabetes-org.csv\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = train['Target']   \n",
    "X_train = train.drop([\"Target\"], axis=1)\n",
    "\n",
    "feat_names = X_train.columns \n",
    "\n",
    "from scipy.sparse import csr_matrix\n",
    "X_train = csr_matrix(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "lr = LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('logloss of each fold is: ', array([0.49442383, 0.50834194, 0.4830274 , 0.43615578, 0.48793522]))\n",
      "('cv logloss is:', 0.4819768328671518)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "loss = cross_val_score(lr, X_train, y_train, cv=5, scoring='neg_log_loss')\n",
    "#%timeit loss_sparse = cross_val_score(lr, X_train_sparse, y_train, cv=3, scoring='neg_log_loss')\n",
    "print ('logloss of each fold is: ',-loss)\n",
    "print ('cv logloss is:', -loss.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise-deprecating',\n",
       "       estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='warn',\n",
       "          n_jobs=None, penalty='l2', random_state=None, solver='liblinear',\n",
       "          tol=0.0001, verbose=0, warm_start=False),\n",
       "       fit_params=None, iid='warn', n_jobs=4,\n",
       "       param_grid={'penalty': ['l1', 'l2'], 'C': [0.1, 1, 10, 100, 1000]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "penaltys = ['l1','l2']\n",
    "Cs = [ 0.1, 1, 10, 100, 1000]\n",
    "tuned_parameters = dict(penalty = penaltys, C = Cs)\n",
    "\n",
    "lr_penalty= LogisticRegression(solver='liblinear')\n",
    "grid= GridSearchCV(lr_penalty, tuned_parameters,cv=5, scoring='neg_log_loss',n_jobs = 4,)\n",
    "grid.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4820287373783356\n",
      "{'penalty': 'l2', 'C': 1}\n"
     ]
    }
   ],
   "source": [
    "# examine the best model\n",
    "print(-grid.best_score_)\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3Xt8VPWd//HXZyYJ4RIFCSD3BOWiXIWAqIiAa2WtRZF6WWut/mrptlqt221rbddV3LZaa1u7uqWWVWlrRauLRaRiVShKC4KIckcE1KAUREARyG0+vz/OSZiEJDOBmUxC3s/HI8453/M9cz5zTObD93y/53vM3REREalPJNMBiIhI06dkISIiCSlZiIhIQkoWIiKSkJKFiIgkpGQhIiIJKVmIiEhCShYiIpKQkoWIiCSUlekAUiU/P98LCgoyHYaISLPy2muvfejunRLVS2uyMLOJwH1AFJjh7nfV2H4NcA+wLSy6391nhNueA0YDr7j7hYmOVVBQwPLly1MYvYjIsc/M3kmmXtqShZlFgQeA84BiYJmZzXH3tTWqPu7uN9TyFvcAbYCvpitGERFJTjr7LEYBm9x9s7uXArOAi5Ld2d1fBD5JV3AiIpK8dCaL7sB7cevFYVlNU8zsTTN70sx6pjEeERE5Qpnu4H4GeMzdS8zsq8BMYEKyO5vZVGAqQK9evdIToYg0eWVlZRQXF3Pw4MFMh9Jk5ebm0qNHD7Kzs49o/3Qmi21AfEuhB4c6sgFw911xqzOAnzTkAO7+IPAgQFFRkR7MIdJCFRcXk5eXR0FBAWaW6XCaHHdn165dFBcXU1hYeETvkc7LUMuAvmZWaGY5wBXAnPgKZtY1bnUSsC6N8YjIMergwYN07NhRiaIOZkbHjh2PquWVtpaFu5eb2Q3AfIKhsw+5+xozmwYsd/c5wI1mNgkoBz4Crqnc38xeBgYA7cysGPiyu89PV7wi0rwpUdTvaM9PWvss3H0eMK9G2W1xy98DvlfHvmenMzYRadku//XfAXj8q2dkOJLmQdN9EPzSVP7iiIgciXbt2lUtT5w4kfbt23PhhbXfT3z99dczbNgwTj31VFq3bs2wYcMYNmwYTz75ZIOOuWLFCp577rmjijtZmR4NJSJyzPn2t7/N/v37+fWvf13r9gceeACArVu3cuGFF7Jy5cojOs6KFStYvXo1EydOPOJYk6WWhYhIip177rnk5eUd0b5vvfUW559/PiNGjGDs2LFs3LgRgFmzZjFo0CCGDh3K+PHjOXDgANOmTePRRx89olZJQ6llISLHlDueWcPa9z9OWG/tB0GdZC5Bn9rtOP7zcwOPOrZkTJ06lRkzZnDSSSexePFibrjhBp5//nnuuOMOFi5cSJcuXdizZw+tW7fmtttuY/Xq1fziF79Ie1xKFiIiTcSePXtYsmQJU6ZMqSorLy8H4KyzzuLqq6/m0ksv5ZJLLmn02JQsROSYkmwLoCmOhnJ38vPza+3D+M1vfsPSpUuZO3cuw4cP5/XXXwdgb8letuzdQuHxR3azXbLUZyGSZqc/PIXTH56SuKIALft8dejQga5duzJ79mwAYrEYb7zxBgCbN29m9OjR3HnnnXTo0IFt27aRl5fHxx/v42BZRdpjU7IQEUmxs88+m0svvZQXX3yRHj16MH9+8vcTz5o1i+nTpzN06FAGDhzI3LlzAbj55psZPHgwgwcPZvz48QwaNIgJEyawYc16Lhk/SR3c0vRU/qtv6bVPZTgSkaZj3759Vcsvv/xyUvsUFBSwevXqamV9+vSpNbnMmTPnsLJOnTrxxxeCVsgp+Sc3JNwGU7IQkRapKfVVNAe6DCUiIgkpWYiISEJKFiIikpCShYiIJKRkISIt08OfDX4kKUoWIiIp0NhTlM+ePZt77rnnqONOlobOAltzfhou6b4BETl6qZqivLy8nKys2r+mJ0+eDMC6DzelIOLE1LIQEUmxo5mifMyYMdx8880UFRVx//3386c//YnTTz+d0047jc985jPs2LEDgBkzZvDNb34TgO987VvcdNNNnHnmmfTp06dqupBUUstCRI4tf74Ftq9KXG/7m8FrMv0WJw6Gf77r6OJqgIqKCpYvXw7A7t27mTRpEmbG9OnTuffee7n77rsP22fHjh0sXryYVatWcdlll1W1PFJFyUJEpIm5/PLLq5bfffddLrvsMrZv305JSQn9+vWrdZ+LL74YM2PIkCFs27Yt5TEpWYjIsSXZFkBli+LaZ9MXyxFq27Zt1fL111/PrbfeygUXXMALL7zAXXfV/vlatWpVtezuKY9JfRYiIk3Y3r176d69O+7OzJkzMxaHkoWISIodzRTlNd1+++1MnjyZkSNH0qVLlxRG2TC6DCUikgKpmqL8lVdeqbY+ZcqUao9ZrXTdddcBwdDZn/zq3mpTlMfHkipKFiLSMjXBvoqmTJehREQkISULERFJSMlCREQSUrIQEZGElCxEpEW69rlrufa5azMdRrOR1mRhZhPNbIOZbTKzW2rZfo2Z7TSzleHPdXHbvmRmb4U/X0pnnCIiR6tyivKVK1dyxhlnMHDgQIYMGcLjjz9+WN1UTFEOsGLFCl5+cVFK4k8kbUNnzSwKPACcBxQDy8xsjruvrVH1cXe/oca+JwD/CRQBDrwW7rs7XfGKiKRCmzZt+O1vf0vfvn15//33GTFiBOeffz7t27evqpPsFOWJrFixgleWvcLZ545NSez1SWfLYhSwyd03u3spMAu4KMl9zwf+4u4fhQniL8DENMUpIpIy/fr1o2/fvgB069aNzp07s3PnzqT3f+uttzj//PMZMWIEY8eOZePGjQDMmjWLQYMGMXToUMaPH8+BAweYNm0ac5+aw+Rxn2twq6Sh0nlTXnfgvbj1YuD0WupNMbOxwEbgZnd/r459u9fc0cymAlMBevXqlaKwRaQ5u/vVu1n/0fqE9SrrJNNvMeCEAXx31HcbHMurr75KaWkpJ510UtL7TJ06lRkzZnDSSSexePFibrjhBp5//nnuuOMOFi5cSJcuXdizZw+tW7fmtttuY9GyV/jeD39Q7Q7udMj0HdzPAI+5e4mZfRWYCUxIdmd3fxB4EKCoqCj10yyKiByhDz74gC9+8YvMnDmTSCS5izh79uxhyZIl1ab3KC8vB+Css87i6quv5tJLL+WSSy5JS8z1SWey2Ab0jFvvEZZVcfddcaszgJ/E7Tuuxr4LUx6hiBxzkm0BVLYoHp74cMpj+Pjjj/nsZz/LD3/4Q0aPHp30fu5Ofn5+rX0Yv/nNb1i6dClz585l+PDhvP7666kMOaF09lksA/qaWaGZ5QBXAHPiK5hZ17jVScC6cHk+8Bkz62BmHYDPhGUiIk1aaWkpkydP5uqrr+bzn/98g/bt0KEDXbt2rXosaiwW44033gBg8+bNjB49mjvvvJMOHTqwbds28vLy+HTfpyn/DLVJW7Jw93LgBoIv+XXAE+6+xsymmdmksNqNZrbGzN4AbgSuCff9CLiTIOEsA6aFZSIiTdoTTzzBokWLeOSRR6qGxDZktNOsWbOYPn06Q4cOZeDAgcydOxeAm2++mcGDBzN48GDGjx/PoEGDmDBhAhvWrOeS8ZOadQc37j4PmFej7La45e8B36tj34eAh9IZn4hIqlROC37VVVdx1VVXJbVPbVOU9+nTp9bnX8yZM+ewsk6dOvHHF4JWyLHewS0ikhHp6Ks4lmm6DxERSUjJQkREElKyEBGRhJQsREQkISULEWmR3vni1bzzxaszHUazoWQhIpICjT1F+ezZs7nnnntSFn8iGjorIpJCqZyivLy8nKys2r+mJ0+eDMC6Dzel+BPUTslCRCSF+vXrV7UcP0V5fLKoz5gxYxg5ciQvv/wyV111FYWFhfzoRz+itLSUTp068fvf/57OnTszY8YMVq9ezVd/cAPf+dq36NOtgGXLlrF9+3buvffeqmSSKkoWInJM2f6jH1GyLvEU5QfXB3WS6bdodcoATrz11gbHciRTlANUVFSwfPlyAHbv3s2kSZMwM6ZPn869997L3Xfffdg+O3bsYPHixaxatYrLLrtMyUJEpDk4kinKK11++eVVy++++y6XXXYZ27dvp6SkpFrLJd7FF1+MmTFkyBC2bdtWa52joWQhIseUZFsAlS2K3r/7bcpjONIpyiu1bdu2avn666/n1ltv5YILLuCFF17grrvuqnWfVq1aVS27p/7xPhoNJSKSQkczRXlt9u7dS/fu3XF3Zs6cmYIIj4yShYhICh3tFOU13X777UyePJmRI0fSpUuXFEbaMLoMJSKSAqmaovyVV16ptj5lypRqj1mtdN111wHB0Nmf/OrealOUV8aSSkoWItIipaOv4limy1AiIpKQkoWIHBPSMQLoWHK050fJQkSavdzcXHbt2qWEUQd3Z9euXeTm5h7xe6jPQkSavR49elBcXMzOnTszHUqj+2Bf+Jl3ltVbLzc3lx49ehzxcZQsRKTZy87OprCwMNNhZMQ1D/8AgKXXPpXW4+gylIiIJKRkISIiCSlZiIhIQkoWIiKSkJKFiIgkpGQhIiIJKVmIiEhCShYiIpJQWpOFmU00sw1mtsnMbqmn3hQzczMrCtdzzOxhM1tlZm+Y2bh0xikiIvVL2x3cZhYFHgDOA4qBZWY2x93X1qiXB9wELI0r/gqAuw82s87An81spLvH0hWviIjULZ0ti1HAJnff7O6lwCzgolrq3QncDRyMKzsVeAnA3XcAe4CiNMYqIiL1SGey6A68F7deHJZVMbPhQE93f7bGvm8Ak8wsy8wKgRFAzzTGKiIi9cjYRIJmFgF+BlxTy+aHgFOA5cA7wN+AilreYyowFaBXr17pClVEpMVLZ8tiG9VbAz3Cskp5wCBgoZltBUYDc8ysyN3L3f1mdx/m7hcB7YGNNQ/g7g+6e5G7F3Xq1CltH0REpKVLZ7JYBvQ1s0IzywGuAOZUbnT3ve6e7+4F7l4ALAEmuftyM2tjZm0BzOw8oLxmx7iIiDSetF2GcvdyM7sBmA9EgYfcfY2ZTQOWu/ucenbvDMw3sxhBa+SL6YpTREQSS2ufhbvPA+bVKLutjrrj4pa3Av3TGZuIiCRPd3CLiEhCShYiIpKQkgVQur8LHotmOgwRkSYrY/dZNBXb9x5k93ufBSvny48sY/yAzowf0Jnu7VtnOjQRkSajxSeLDm2zad/9eUo+7cHGHXm8uH4HAP275AWJo38nRvTuQFZUjTARablafLJolRWlVbtiWrUrZtE1/87bOz9lwfodLNiwgxkvb2b6X98mLzeLsf06MaF/Z87p34n8dq0yHbaISKNq8ckinplxcud2nNy5HV8Z24dPDpaxeNOHvLR+Bws27OTZNz/ADIb0aM/4/p2YMKAzg7odTyRimQ5dRCStGpwswjmd2rn7x2mIp0nJy81m4qCuTBzUlVjMWfvBx2Hi2MF9L77FL154i/x2rRgXJo4xffM5Ljc702GLiKRcUsnCzP4A/CvBZH7LgOPM7D53vyedwTUlkYgxqPvxDOp+PDee25dd+0pY9NZOXlq/k+fXbOfJ14rJihhFBR0Y378zEwZ05uTO7TBTq0NEmr9kWxanuvvHZvYF4M/ALcBrQItJFjV1bNeKyaf1YPJpPSiviPH6e3uCVsf6Hfz4z+v58Z/X06ND66rEMbpPR1rnaHiuiDRPySaLbDPLBi4G7nf3MjPzNMbVrGRFI4wsOIGRBSfw3YkDeH/PARZu2MlL63fw5GvF/G7JO7TKinDmSR3DEVad6XlCm0yHLSKStGSTxa+BrQQPJVpkZr2BY77P4kh1a9+aK0/vxZWn9+JgWQWvbvmoqq9jwZ/WAGvo27ldVeIoKuhAtobmikgTllSycPdfAr+MK3rHzManJ6RjS252lLH9OjG2XyduZyCbd+5jwYadLFi/g4cXb+HBRZvJa5XF2f3yGde/M+P6d6JzXm6mwxYRqSbZDu6bgIeBT4AZwGkE/RbPpy+0Y1OfTu3o06kdXx5TyL6SchZv+rDqvo55q7YDMKTH8YwL+zqGdNfQXBHJvGQvQ/0/d7/PzM4HOhA8X+J3KFkclXatsjh/4ImcP/BE3IOhuZV9Hfe/9Ba/fPEtOrbN4Zz+nRjfvzNj+3Xi+NYamisijS/ZZFH5T9sLgN+FDzE6Zv6527vs7UyHgJkxsNvxDOx2PNePP5ndn5aGQ3N38NL6Hfzfim1EI8aIXh0YPyBodfTroqG5ItI4kk0Wr5nZ80Ah8D0zywNi6QtLOrTN4aJh3bloWHcqYs7K93azYH2QPO5+bj13P7ee7u1bMy5sdZx5ckfa5OiGfBFJj2S/Xb4MDAM2u/t+M+sIXJu+sCReNGKM6H0CI3qfwL+f35/tew+ycEPQ4pj9+jYeXfouOVkRRvfpyIT+nZgwoAu9OmporoikTrKjoWJm1gO4Mrzs8Vd3fyatkUmdTjw+lytG9eKKUb0oKa9g2ZbdwbDc9Tu4/Zm13P7MWvp0asuE/sF06yMLTiAnS0NzReTIJTsa6i5gJPBoWHSjmZ3h7remLTJJSqusKGP65jOmbz7/ceGpbP3w0yBxbNjJb//+DjNe2UK7VlmMOTmf8QOCS1adj9PQXBFpmGQvQ10ADHP3GICZzQReB5QsmpiC/LZcm1/ItWcVsr+0nMWbdlW1Op5bEwzNHdjtOCYM6My4/p0Z1rM9UQ3NFZEEGtIj2h74KFw+Pg2xSIq1ycnivFO7cN6pXXB3NvzjE15av4OF63fyPwvf5r9f2kSHNtmc068T4wd05px+nWjfJifTYYtIE5Rssvgx8LqZLSAYRjuW4Ka8Zm9vyV425lRgwOdmf47WWa1pndWa3KxccqO55GblVluvXK6tTuus1lXrVXWiuUQjmZ9A0MwYcOJxDDjxOL4+7mT27i9j0VvBneQLN+7k6ZXvEzEYHg7NHd+/M6d0zdPQXBEBku/gfszMFhL0WwB81923py2qRhSxCMfFjBjQ/4T+HCw/yMHyg+wr28fOAzur1g+WH+RAxQHKY+UNPkZ2JLt6kolPKNG45BRXVi3h1CirTFrx+2VHGnaz3vFtsvnc0G58bmg3KmLOm8V7wjvJd3LP/A3cM38DJx6XW9XPcdbJ+bRtpaG5Up17MJ+o49XK3B3HCTcTC7d77FD9yj1iYaVYcJWbWCz4B8qeA/vSG/uR7HMEO3kdO/kRRXC4WCzCkX2ahqn3r9/MhtcoKg5fu5lZN3dfkZ6wGk9eTh4nlgcjhX56zk8T1i+LlVFSXsKB8gNVCaQqoVQcPFRez/bKbQfLD7K3ZO+h+hVBWUlFSYM/R5ZlVUsmuVm5tMlqc1hZZbJqnd36sLJhA3IZPSiXkpJ8VhUfYMXWT3lm9ToeW76BnEgrRhV2ZkL/LpSXHkckaz8fHywjFnNiDhUxJ+ZORcwpr4hRFiunrKKc8lgFpRUVlMcqKIuVUx4rp7yicjkW1PGKoMzLicUqKIsF9SsqX70i2K9yuSJGhVdQ4eVUxCqXK8LlQ2WxynKPEfMKYh5sD14r8MpXKrcHdWIEr5Xl8a9ODK/cRvUyiOEWAw9enQrAiXk54Ax+6AwO/6P2w5ctbvWwyZ1r+1KI+6Ku9/1DtU4Ynbis0SaaDr+Vzn7ijMY5XnOXBbGDJzbGYep1bz3bHJiQwliahexINtk52bTLaZe2Y1TEKiipKKmWQCoTSnxZVdKpqH17ZdmHBz48rP6B8gNJxWK9IS9cfsONlRuz8axswDjz0TFgwZckFgu/hGKN96XSQO6GYeAR4NCPhetGzdcowR5RLCw3IphFiJJFFjmYRYgQrXqNWFAnYpXLUd79+AMMo6B9jzASi/svQUxVBVatLP4i4KFLgodKDePQ7nXtW3ed+ANUbTM7vKxqPVJtv6qtZtVjPfzTVb1vtXevERdmLCl+HTDO6HEaqVL3xdQjuMx6RLuk73Lu4uLlREj/0Ph6k4W7a2bZDIhGorSJtKFNdvpurHN3SipKqpLH/vL9h11yq61FtHPfPmavXYx7hKFdTiFqUSIWJWoRopEsohYJ16NEIxGywrKsSFAvKxIlGgm2Z1sW0UiEaCRKdiQrfD1UJyuSVbWeFckiOxpXFq2sm0V2NCjLiWZVledEs4JjR4Iv7Wj45Z0Jpz88BXDmXvnzjBy/uQnOF/zmom9nOJLmofJ8pVuy91lcUkvxXmCVu+9IbUjSGMys6hJVQ83fFPxyPjr57lSHJSJNVEOm+zgDWBCujyN4rGqhmU1z99+lITYREWkikk0WWcAp7v4PADPrAvwWOB1YRDBduYiIHKOSvYjbszJRhHaEZR8BZXXtZGYTzWyDmW0yszrvyzCzKWbmZlYUrmeb2UwzW2Vm68zse0nGKSIiaZBsy2Khmc0F/hiufz4sawvsqW0HM4sCDwDnEQy5XWZmc9x9bY16ecBNwNK44kuBVu4+2MzaAGvN7DF335pkvCIikkLJtiyuJ3is6rDwZyZwvbt/Ws+IqVHAJnff7O6lwCzgolrq3QncDRyMK3OgrZllAa2BUuDjJGMVEZEUSypZeHAL4ivAS8CLwCKv67bEQ7oD78WtF4dlVcKb/nq6+7M19n0S+BT4AHgX+Gl4yUtERDIgqWRhZpcBrxJcfroMWGpmnz+aA5tZBPgZ8K1aNo8CKoBuBE/n+5aZ9anlPaaa2XIzW75z586jCUdEROqRbJ/F94GRlfdUmFkn4AWCFkBdtgE949Z7hGWV8oBBBH0fACcCc8xsEnAl8Jy7lwE7zGwxUARsjj+Auz8IPAhQVFTUNG8bFhE5BiTbZxGpcfPdriT2XQb0NbNCM8sBrgDmVG50973unu/uBe5eACwBJrn7coJLTxMAwk700cD6JGMVEZEUS7Zl8ZyZzQceC9cvB+bVt4O7l5vZDcB8IAo85O5rzGwasNzd59Sz+wPAw2a2hmAmlofd/c0kY22Y8lJu//B9/t66HZTuhxw9u1pEpKZkpyj/tplNAc4Kix5099lJ7DePGknF3W+ro+64uOV9BMNn02/fdrIcvrx3F/x8IIyaCqO+Am3zG+XwIiLNQdIPKHD3p4Cn0hhLZrTvxQ/yuzKgtIRpHYfBX++CxffBaV+AM66HEw7rVxcRaXESPc/iE2qf6N4IRtQel5aoGtk1fywDIvDCLNixHv7+3/DaTFj+EJwyCc66EbqPyHSYIiIZk2iK8rz6th+TOg+Aix6A8T+ApdNh+cOw9mkoOBvOvBH6ngd61KiItDCZmeC/OTiuK5x3B9y8Gj7zX/DRZvjDpfCrM2HlH6C8NNMRiog0GiWLRHKPgzO/ATeuhIunB2VPfw3uGwqLfwkHNQuJiBz7lCySlZUDw/4FvvY3+MKT0PEk+Mt/BCOo/nIbfPxBpiMUEUkbJYuGMgv6La6ZC19ZACefC3/7b/jFYHj6+qCDXETkGKNkcTS6D4dLH4FvrIAR18Dqp+B/Toc/XA5bF0PCuRZFRJoHJYtUOKEQPvtTuHkNjPseFC+DRy6AGf8Ea/8EsYpMRygiclSULFKpbUcYdwt8czVc8FPY/yE8cTXcXwTL/hfKDmQ6QhGRI6JkkQ45bYIpQ76xIrhMldsenv03+Pkg+OtPYL8ezSEizYuSRTpFojBwMnzlJfjS3KCPY8EPgxFU874Du9/JdIRHpHfZ2/QuezvTYYhII0p6big5CmZQeHbw84+1weip5Q/Bshkw8OLgzvBuwzIdZdKueSK8IXFqZuNoLv7z0XXBwrWZjaO50PlqmMY6X2pZNLYup8LkX8FNb8AZX4eNz8OD58DMSbDpBY2gEpEmSckiU47vHkwj8m9r4J/ugA83wu+nwPSz4Y3HoaIs0xGKiFRRssi03ONhzDeDlsZFD0CsDGZPhfuGwd8fgJJPMh2hiIiSRZOR1QpOuwq+9ne48gno0Bvm3xp0hr9wB3zyj0xHKCItmJJFUxOJQL/z4dp5cN2LUHgOvPJz+MUgmPMN+PCtTEcoIi2QkkVT1qMILv8dfOO1oNXx5hNw/0h47Ep4d0mmoxORFkTJojnoeBJc+PPgzvCx34Z3/wYPnQ//+xlYNxdisUxHKCLHOCWL5qRdJ5jw/WAOqn/+CXzyATz+BXhgFLz2CJQdzHSEInKMUrKA5ndvQ05bOP2r8I3X4fMPBdOLPHNTME36op/Cgd2ZjlBEjjEt/g7uin2fUrANSrPgva9fT05hATkFBbQqLCSnsJDoCSdgTfWZ29EsGDQFBl4CWxbB4vvgpTvh5Z/BiC/B6K9D+56ZjlJEjgEtPllQXsZHeVnklDtl773Lpy+/jJcduiEukpdHTmEhOQW9gwRSUBCs9+5NpHXrDAYexwz6nBP8bF8VTCfy6oOw9NdBMjnrRjhxcKajFJFmrMUni2j79uxsnwPAqGeewSsqKHv/fUq3bqV0yxZKt26lZMsW9i9bzsdznqm2b1bXrrQqDFoiOQWFQauksJDsrl2xaLTxPwwESeGSB2HCf8CSX8GKmbDqCThpQjAHVZ9xQXIREWmAFp8sarJolJyePcnp2RPOPrvattj+/ZS+805VAgkSylb2znmG2L59h94jJ4ec3r2CBFLZEikMkkpWhw6N80Ha94SJP4Jzvg3LH4al0+F3F8OJQ+Csm+DUi4PLWCIiSdC3RQNE2rQh95RTyD3llGrl7k7Frl2UbtlCSZhASrdupWTTJj5ZsADKy6vqRtu3P5RACgrIKQz6R7J79SLSqlXqg27dAc7+NzjjenjzcVj8S3jqy/DiHTD6ehj+xaDDXESkHkoWKWBmZOXnk5WfT5uRI6tt87IyyrZtC1si71Rd2vr0lVfYO3t2/JuQ3b17XCLpXdXJntWlCxY5yoFrWa1g+NUw7CrY+FzQGf7cd2Hhj4MHNY36ajA0V0SkFkoWaWbZ2WGfRsFh2yr27auWQCpf96xYge/ff+g9cnOr3qOyJVKZVKJ5eQ0LKBKBARcEP+8uhb/9Mhhuu/iXMOxKOPMbwU2AIiJxlCwyKNquHa0HDaT1oIHVyt2d8h07wstZWyjdspWSrVs4uG4tnzz/fLU7tqMdOx423DenoICcHj2wnJz6A+h1OvR6NJhv6u/3w8o/BDf3nXIhnHkT9BxZ//4i0mKkNVmY2UTgPiB2qJBQAAANMElEQVQKzHD3u+qoNwV4Ehjp7svN7AvAt+OqDAGGu/vKdMbbVJgZ2V26kN2lC21Hn15tm5eWUvree1Utkco+kn0LFrL3yacOVYxGye7RnVbVOtmD5azOnarfO5LfFz53H4z/fjDcdtlvYN0z0OvMYNht3/ODFomItFhpSxZmFgUeAM4DioFlZjbH3dfWqJcH3AQsrSxz90eBR8Ptg4GnW0qiSMRycmh10km0OunwS0UVe/cGSaRqtFZwievTJUvwkpKqepE2bQ5LIDmFBeT0LiB67n/AmJvh9d8Fz9N47ArI7x9cnhpyWdD3ISItTjpbFqOATe6+GcDMZgEXAWtr1LsTuJvqLYl4/wLMSleQx5Lo8cfTeuhQWg8dWq3cYzHKt2+vNty3dMsWDqxcycfz5lWb7iSrc+dDCaT3v5HT4X1affAs2U/fgL30XzD6X8mKxiivUEtDpCVJZ7LoDrwXt14MVLumYmbDgZ7u/qyZ1ZUsLidIMocxs6nAVIBevXoddcDHKotEyO7Wjexu3eCss6ptix08SOm771YN9y3dsoXSLVv4+LnniO3de6hiVk9yjo+S8+f7GNy2ggqMndeOCC9nWXCjX/xPVRmABfUO21ZjHTCLQKRGnarlSPB+YblVK6+xDFXrFr8tOCG1xhuMOLPw+LW8Z+VxLa5e1bZI1ftU7R9+jm5ZJeDwySO1XoWVGrpFS3Dgk5l3ZzqUZqFrtIQy0n+jbcY6uC34i/sZcE09dU4H9rv76tq2u/uDwIMARUVFzWw2wKYhkptLbr9+5Pbrd9i28t27q1ohweWtLZRsXM++t94DNz5hfy3vKDW1IhhoUHzXzAxH0jxUna8fP5LZQJqJXHLIa1+WuOJRSmey2AbEz2LXIyyrlAcMAhaGna0nAnPMbJK7Lw/rXAE8lsYYpR5ZHTqQ1aEDbYafVq183rnBTYn/PH9VUOBedSmrKmPHlR22XPVSvaz2enH/BkhU79DB66yHe1jXwWPBTyxW67J7LKgXc/CKsNwPrxeLBcd0h1hFcHivqKq74s5vAjD8+/cmPOcCr//wWwCc9v2fZjiS5uH1H/47FRgnp/k46UwWy4C+ZlZIkCSuAK6s3Ojue4H8ynUzWwj8e2WiCFselwHV59yQzKu8ZJR1+K+PZp063J6K7wCQO7bWq6lSw+5p3wMgd+zFGY6kedg97dZGOU7aeindvRy4AZgPrAOecPc1ZjbNzCYl8RZjgfcqO8hFRCRz0tpn4e7zgHk1ym6ro+64GusLgdHpik1ERJKn8Y8iIpKQkoWIiCSkZCEiIgkpWYiISEJKFiIikpCShYiIJKTnWQBzvjUEgMkZjkNEpKlSy0JERBJSshARkYSULEREJCElCxERSUjJQkREElKyEBGRhJQsREQkISULERFJSMlCREQSUrIQEZGElCxERCQhzQ0lDXbHF04B4IIMx9Fc6Hw1jM5XwzTW+VLLQkREElKyEBGRhJQsREQkISULERFJSMlCREQSUrIQEZGElCxERCQhJQsREUlIyUJERBJSshARkYQ03Qfw8MSHMx2CiEiTltaWhZlNNLMNZrbJzG6pp94UM3MzK4orG2JmfzezNWa2ysxy0xmriIjULW0tCzOLAg8A5wHFwDIzm+Pua2vUywNuApbGlWUBvwe+6O5vmFlHoCxdsYqISP3S2bIYBWxy983uXgrMAi6qpd6dwN3AwbiyzwBvuvsbAO6+y90r0hiriIjUI53JojvwXtx6cVhWxcyGAz3d/dka+/YD3Mzmm9kKM/tObQcws6lmttzMlu/cuTOVsYuISJyMjYYyswjwM+BbtWzOAsYAXwhfJ5vZuTUrufuD7l7k7kWdOnVKa7wiIi1ZOpPFNqBn3HqPsKxSHjAIWGhmW4HRwJywk7sYWOTuH7r7fmAeMDyNsYqISD3SmSyWAX3NrNDMcoArgDmVG919r7vnu3uBuxcAS4BJ7r4cmA8MNrM2YWf3OcDaww8hIiKNIW3Jwt3LgRsIvvjXAU+4+xozm2ZmkxLsu5vgEtUyYCWwopZ+DRERaSRpvSnP3ecRXEKKL7utjrrjaqz/nmD4rIiIZJim+xARkYSULEREJCElCxERSUgTCUqDLb32qUyHICKNzNw90zGkRFFRkS9fvjzTYYiINCtm9pq7FyWqp8tQIiKSkJKFiIgkpGQhIiIJKVmIiEhCShYiIpKQkoWIiCSkZCEiIgkpWYiISEJKFiIiktAxcwe3me0E3jmKt8gHPkxROKmkuBpGcTWM4mqYYzGu3u6e8LnUx0yyOFpmtjyZW94bm+JqGMXVMIqrYVpyXLoMJSIiCSlZiIhIQkoWhzyY6QDqoLgaRnE1jOJqmBYbl/osREQkIbUsREQkoRabLMzsUjNbY2YxM6tzFIGZTTSzDWa2ycxuaYS4TjCzv5jZW+FrhzrqVZjZyvBnThrjqffzm1krM3s83L7UzArSFUsDYrrGzHbGnZ/r0h1TeNyHzGyHma2uY7uZ2S/DuN80s+FNJK5xZrY37nzd1khx9TSzBWa2NvxbvKmWOo1+zpKMq9HPmZnlmtmrZvZGGNcdtdRJ39+ju7fIH+AUoD+wECiqo04UeBvoA+QAbwCnpjmunwC3hMu3AHfXUW9fI5yjhJ8f+DowPVy+Ani8CcR0DXB/Bn6nxgLDgdV1bL8A+DNgwGhgaROJaxwwNwPnqyswPFzOAzbW8v+y0c9ZknE1+jkLz0G7cDkbWAqMrlEnbX+PLbZl4e7r3H1DgmqjgE3uvtndS4FZwEVpDu0iYGa4PBO4OM3Hq08ynz8+3ieBc83MMhxTRrj7IuCjeqpcBPzWA0uA9mbWtQnElRHu/oG7rwiXPwHWAd1rVGv0c5ZkXI0uPAf7wtXs8Kdmp3Pa/h5bbLJIUnfgvbj1YtL/S9PF3T8Il7cDXeqol2tmy81siZmlK6Ek8/mr6rh7ObAX6JimeJKNCWBKeNniSTPrmcZ4GiITv0/JOiO8vPFnMxvY2AcPL5ecRvCv5XgZPWf1xAUZOGdmFjWzlcAO4C/uXuf5SvXfY1Yq3qSpMrMXgBNr2fR9d/9TY8dTqb644lfc3c2sruFqvd19m5n1AV4ys1Xu/naqY22mngEec/cSM/sqwb+0JmQ4pqZsBcHv0z4zuwB4GujbWAc3s3bAU8A33f3jxjpuIgniysg5c/cKYJiZtQdmm9kgd6+1LyrVjulk4e7/dJRvsQ2I/1dpj7DsqNQXl5n9w8y6uvsHYXN7Rx3vsS183WxmCwn+9ZPqZJHM56+sU2xmWcDxwK4Ux9GgmNw9/vgzCPqBmoK0/D4drfgvQnefZ2b/Y2b57p72OZDMLJvgC/lRd/+/Wqpk5JwliiuT5yw85h4zWwBMBOKTRdr+HnUZqn7LgL5mVmhmOQQdRmkbeRSaA3wpXP4ScFgLyMw6mFmrcDkfOAtYm4ZYkvn88fF+HnjJw961NEkYU41r2pMIrjk3BXOAq8MRPqOBvXGXHDPGzE6svK5tZqMIvhfSmfArj2vA/wLr3P1ndVRr9HOWTFyZOGdm1ilsUWBmrYHzgPU1qqXv77Exe/Ob0g8wmeD6ZwnwD2B+WN4NmBdX7wKC0RBvE1y+SndcHYEXgbeAF4ATwvIiYEa4fCawimAk0Crgy2mM57DPD0wDJoXLucAfgU3Aq0CfRjhHiWL6MbAmPD8LgAGN9Dv1GPABUBb+bn0Z+FfgX8PtBjwQxr2KOkbhZSCuG+LO1xLgzEaKawxBB+2bwMrw54JMn7Mk42r0cwYMAV4P41oN3BaWN8rfo+7gFhGRhHQZSkREElKyEBGRhJQsREQkISULERFJSMlCREQSUrIQaQAz25e4Vr37PxnedY+ZtTOzX5vZ22b2mpktNLPTzSzHzBaFN1WJNAlKFiKNJJw/KOrum8OiGQQT/PV19xHAtUC+BxMkvghcnplIRQ6nZCFyBMI7iu8xs9VmtsrMLg/LI+HUD+steB7JPDP7fLjbFwjvyDezk4DTgR+4ewzA3be4+7Nh3afD+iJNgpq5IkfmEmAYMBTIB5aZ2SKCqVcKgFOBzgRTjTwU7nMWwd3UAAOBlR5MDFeb1cDItEQucgTUshA5MmMIZratcPd/AH8l+HIfA/zR3WPuvp1gupFKXYGdybx5mERKzSwvxXGLHBElC5HGc4Bg7h4I5hUaambReuq3Ag6mPSqRJChZiByZl4HLw4fRdCJ4dOmrwGKCBy9FzKwLweM3K60DTgbw4Nkjy4E74mYvLTCzz4bLHYEP3b2ssT6QSH2ULESOzGyC2T/fAF4CvhNednqKYGbXtcDvCR6Sszfc51mqJ4/rCJ6EuMnMVgOPcOj5JePD+iJNgmadFUkxM2vnwRPUOhK0Ns5y9+3hMwgWhOt1dWxXvsf/Abe4+8ZGCFkkIY2GEkm9ueFDanKAO8MWB+5+wMz+k+A5ye/WtXP4UKenlSikKVHLQkREElKfhYiIJKRkISIiCSlZiIhIQkoWIiKSkJKFiIgkpGQhIiIJ/X8bf4fVF60HqAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot CV误差曲线\n",
    "test_means = grid.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid.cv_results_[ 'std_test_score' ]\n",
    "train_means = grid.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = grid.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "# plot results\n",
    "n_Cs = len(Cs)\n",
    "number_penaltys = len(penaltys)\n",
    "test_scores = np.array(test_means).reshape(n_Cs,number_penaltys)\n",
    "train_scores = np.array(train_means).reshape(n_Cs,number_penaltys)\n",
    "test_stds = np.array(test_stds).reshape(n_Cs,number_penaltys)\n",
    "train_stds = np.array(train_stds).reshape(n_Cs,number_penaltys)\n",
    "\n",
    "x_axis = np.log10(Cs)\n",
    "for i, value in enumerate(penaltys):\n",
    "    #pyplot.plot(log(Cs), test_scores[i], label= 'penalty:'   + str(value))\n",
    "    plt.errorbar(x_axis, -test_scores[:,i], yerr=test_stds[:,i] ,label = penaltys[i] +' Test')\n",
    "    plt.errorbar(x_axis, -train_scores[:,i], yerr=train_stds[:,i] ,label = penaltys[i] +' Train')\n",
    "    \n",
    "plt.legend()\n",
    "plt.xlabel( 'log(C)' )                                                                                                      \n",
    "plt.ylabel( 'logloss' )\n",
    "plt.savefig('LogisticGridSearchCV_C.png' )\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cPickle\n",
    "\n",
    "cPickle.dump(grid.best_estimator_, open(\"L1_org_logloss.pkl\", 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
